const fs = require("fs");
const path = require("node:path");
const schedule = require("node-schedule");
const moment = require("moment");
const { startPlaywright} = require("./use_playwright");
const { startSelenium } = require("./use_selenium");
const {transform} = require("./excel_processor");
const {getProxies} = require("./getProxies")
const colors = require('colors');
let config = require("./config/scrapy.config.json");

const { assert } = require("console");
const { waitForSeconds, TaskEndEvent } = require("./common");
let defaultZipCode = "90001";

var retryCount = 0;

const startJob = async (zipcode,keyword,tasknumber,total,usePlaywright=true,batchDir)=>{

    console.log(`start task: ${tasknumber}/${total}`)

    let source = usePlaywright ? "playwright":"selenium"

    console.log(`will use source:${source} for ${keyword}`);

    if(!keyword){
        console.error(new Error("keyword is null or empty"))
    }

    let dt = moment().format("HH_mm_ss");

    const commonPath = path.resolve(batchDir,dt+"_"+keyword).replace(/\s/,"_");

    let promise = undefined;

    const proxies = await getProxies()

    if(!usePlaywright){
        console.log("start as selenium browser task")
        promise = startSelenium(zipcode,keyword,proxies);
    }else{
        console.log("start as playwright browser task")
        promise = startPlaywright(zipcode,keyword,proxies)
        // promise = startPlaywrightNew(zipcode,keyword)
    }

    const htmlPath = commonPath+".html"
    let dataAsinsPath = commonPath+".csv";

    return new Promise(async(resolve,reject)=>{
        console.log("start task...")
        promise.then(async(r)=>{
            const {html,dataitems} = r;
            console.log(`received result: ${JSON.stringify(r)}`)
            await waitForSeconds(1);
            // console.clear()
            fs.writeFileSync(htmlPath,html);
            let its = (dataitems||[]).filter((i)=>i.dataAsin !== '');
            let finalContent = its.map((item)=>{
                let title = (item.title||"").replace(/\n/g," ").replace(/,/g," ");
                let amount = item.amount||""
                if(item.dataAsin){
                    return item.dataAsin+","+title+","+amount;
                }else{
                    return "";
                }
            }).join("\n");
            
            fs.writeFileSync(dataAsinsPath,finalContent);
                        
            retryCount = 0;
        }).catch(async (e)=>{
            console.error(e);
            retryCount ++;
            console.log(`retrying ... ${zipcode} --- ${keyword}, the ${retryCount} times`)
            let changetoSelenium = false;
            if(retryCount < config.maxRetryTimes){

                assert(config.maxRetryTimes > config.whenSwitchToSelenium);

                if(retryCount >= config.whenSwitchToSelenium){
                    changetoSelenium = true;
                }
                return await startJob(zipcode,keyword,tasknumber,total,!changetoSelenium,batchDir);
            }else{
                console.error(e);
            }
        }).finally(()=>{
            console.log("come here");
            resolve(true)
        })
      
    })
}

const stats = (batchDir,total)=>{
    const outputFormat = ".csv";
    const csvfiles = []
    const readAll = (fp)=>{
        const filenames = fs.readdirSync(fp,{"recursive":true})
        filenames.forEach((f)=>{
            const p = path.join(fp,f);
            if(fs.statSync(p).isDirectory()){
                readAll(p)
            }else{
                if(p.endsWith(outputFormat) && !csvfiles.includes(p)){
                    csvfiles.push(p);
                }
            }
        })    
    }
    readAll(batchDir);
    const outputs = csvfiles.filter((name)=> name.endsWith(outputFormat) && fs.statSync(path.resolve(batchDir,name)).isFile())
    console.log(`expect ${total}, actual ${outputs.length}`.yellow)
    if(outputs.length >= total){
        console.log(`--------------------------------END-------------------------------------`.green);
    }else{
    }    
}

const start = async ({usePlaywright=false})=>{

    try{
        await transform();
    }catch(e){
        return        
    }
    
    let querys = require("../input/query.json");

    let day =  moment().format("yyyy_MM_DD");
    
    let outputDir = path.resolve(__dirname,'..',"output",day);

    if(!fs.existsSync(outputDir)){
        fs.mkdirSync(outputDir,{recursive:true})
    }
    const subdirs = fs.readdirSync(outputDir,{recursive:false});

    const dirnumbers = subdirs.map((num=>parseInt(num))).filter((num)=>!isNaN(num))

    let index = 1;

    if(subdirs.length > 0){
       index = Math.max(...dirnumbers,index)+1
    }

    var batchDir = path.resolve(outputDir,index.toString())

    if(!fs.existsSync(batchDir)){
        fs.mkdirSync(batchDir,{recursive:true});
    }


    let currentTaskCount = Math.max(1,config.concurrentTaskNumber);
    
    let taskGroups = [];
    
    let tasks = []
    let total = 0;

    for (let i=0; i<querys.items.length; i++){
        
        let query = querys.items[i];
        let zipcode = query["zipcode"]||defaultZipCode;
        let keywords = query['keywords']||[];
        /**@type Array */
        if(taskGroups.length > 0){
            let lastTasks = taskGroups[taskGroups.length - 1]
            if (lastTasks.length < currentTaskCount){
                tasks = lastTasks;
            }
        }
        for(let j=0; j<keywords.length; j++){
            
            total ++;
            let keyword = keywords[j];
            let item = {zipcode,keyword,taskNumber:total}
            tasks.push(item)
            
            if(tasks.length == currentTaskCount || (i==querys.items.length -1 && j== keywords.length -1)){
                taskGroups.push({
                    zipcode,
                    tasks:[...tasks]
                });
                tasks = []
            }
        }
    }    

    for (let idx = 0; idx< taskGroups.length; idx++){
        try{
            let tasks = taskGroups[idx].tasks;
            let zipcode = taskGroups[idx].zipcode;
            // console.clear();
            console.log(`start task group for batch ${idx+1}`);
            let zipDir = path.resolve(batchDir,zipcode);
            if(!fs.existsSync(zipDir)){
                fs.mkdirSync(zipDir);
            }

            let ptasks = tasks.map((task)=>{
                return startJob(task.zipcode,task.keyword,task.taskNumber,total,usePlaywright,zipDir)
            });

            if(ptasks && ptasks.length > 0){
              const result = await Promise.allSettled(ptasks);  
            }
            console.log(`complete task group count ${idx+1}`.green)
            // console.clear()
        }catch(e){
            console.log(`err: ${e.toString()}`.red);
        }finally{
        }
        stats(batchDir,total);
    }
}


start({usePlaywright:config.usePlaywright});
/* * * * * */
//second minute hour day month (day of week)

//轮询任务
// schedule.scheduleJob("scrapy task","0 20 14 * * *",()=>{
//     console.log("start scapy task.")
//     start({usePlaywright:true})
// })
